import numpy as np
from sklearn.preprocessing import MinMaxScaler,StandardScaler
from sklearn.decomposition import PCA
from sklearn.manifold import TSNE

path='./data_task1'
name='data_part0'
target=name+'_nominmaxscaler'
target2=name+'_normalizing'
target4=name+'_partition'
target3=name+'_pca'
numpca=25
numpca2=10
numpca3=20
numpca4=30
eps=0.001
kk=1
limit=5
wea=40 #40
count=35 #20
weekday=7 #7
kk=6

def data_clean(datas,labels,tests):
    lt=len(datas)
    lw=len(labels)
    lv=len(datas[0])-weekday
    print lv
    lv2=len(labels[0][0])
    s=[0]*400
    tot=[0]*1000
    aver=[0.0]*400
    aver2=[0.0]*400
    s2=[0]*400
    for i in range(0,lv):
        ss=0
        for j in range(0,lt):
            if datas[j][i]>eps:
                aver[i]+=datas[j][i]
                ss+=1
        aver[i]/=ss
    for k in range(0,lw):
        for i in range(0,lv2):
            ss=0
            for j in range(0,lt):
                if labels[k][j][i]>eps:
                    aver2[k*lv2+i]+=labels[k][j][i]
                    ss+=1
            aver2[k*lv2+i]/=ss
    for i in range(0,lt):
        t=0
        for j in range(0,lv):
            if datas[i][j]<eps and j<lv-wea:
                t+=1
                continue
#            if i>0 and i<lt-1:
#                if datas[i-1][j]>eps and datas[i+1][j]>eps:
#                    if abs(datas[i][j]-datas[i-1][j])>aver[j]*kk and abs(datas[i][j]-datas[i+1][j])>aver[j]*kk:
#                        t+=1
#                        continue
            if datas[i][j]>aver[j]*limit:
                datas[i][j]=aver[j]*limit
                t+=1
        s2[i]=t
        for k in range(0,lw):
            tt=0
            for j in range(0,lv2):
                if labels[k][i][j]<eps:
                    tt+=1
                    continue
                if labels[k][i][j]>aver2[k*lv2+j]*limit:
                    labels[k][i][j]=aver2[k*lv2+j]*limit
                    tt+=1
            t+=tt
        s[i]=t
        tot[t]+=1

    m=1000
    for i in range(1,1000):
        tot[i]+=tot[i-1]
        if tot[i]>count:
            m=i
            break
    d=[]
    l=[]
    for i in range(0,lw):
        l.append([])
    for i in range(0,lt):
        if s[i]<=m or i>=lt-7:
            print i,s[i],s2[i]
            dd=[]
            dd.extend(datas[i])
            ll=[]
            for k in range(0,lw):
                ll.append([])
                ll[k].extend(labels[k][i])
            for j in range(0,lv):
                if dd[j]<eps and j<lv-wea:
                    ave=0.0
                    ss=0
                    for ii in range(i-4,i+5):
                        if ii>=0 and ii<lt and datas[ii][j]>eps:
                            ave+=datas[ii][j]
                            ss+=1
                    if ss==0:
                        ave=aver[j]
                    else:
                        ave/=ss
                    dd[j]=ave
#                else:
#                    if i>0 and i<lt-1:
#                        if datas[i-1][j]>eps and datas[i+1][j]>eps:
#                            if abs(dd[j]-datas[i-1][j])>aver[j]*kk and abs(dd[j]-datas[i+1][j])>aver[j]*kk:
#                                dd[j]=(datas[i-1][j]+datas[i+1][j])/2
            for k in range(0,lw):
                for j in range(0,lv2):
                    if ll[k][j]<eps:
                        ave=0.0
                        ss=0
                        for ii in range(i-4,i+5):
                            if ii>=0 and ii<lt and labels[k][ii][j]>eps:
                                ave+=labels[k][ii][j]
                                ss+=1
                        if ss==0:
                            ave=aver2[k*lv2+j]
                        else:
                            ave/=ss
                        ll[k][j]=ave
#                    else:
#                        if i>0 and i<lt-1:
#                            if labels[k][i-1][j]>eps and labels[k][i+1][j]>eps:
#                                if abs(ll[k][j]-labels[k][i-1][j])>aver2[k*lv2+j]*kk and abs(ll[k][j]-labels[k][i+1][j])>aver2[k*lv2+j]*kk:
#                                    ll[k][j]=(labels[k][i-1][j]+labels[k][i+1][j])/2
            d.append(dd)
            for k in range(0,lw):
                l[k].append(ll[k])
    for i in range(0,lv):
        for j in range(0,len(tests)):
            if tests[j][i]<eps and i<lv-wea:
                tests[j][i]=aver[i]
            if tests[j][i]>=aver[i]*limit:
                tests[j][i]=aver[i]*limit
    print len(d)
    return (d,l,tests)

def part(datas):
    datas2=[]
    l=len(datas[0])-wea-weekday
    print l/kk,wea+weekday
    for i in range(0,kk):
        newd=[]
        for data in datas:
            line=[]
            for j in range(0,l/kk):
                line.append(data[j*kk]+i)
            if wea+weekday>0:
                line.extend(data[-wea-weekday:])
            newd.append(line)
        datas2.append(newd)
    return datas2
            
    

traindata=np.load(path+'/train'+name+'.npz')
testdata=np.load(path+'/test'+name+'.npz')
datas=traindata['datas']
labels=traindata['labels']
tests=testdata['tests']
(datas,labels,tests)=data_clean(datas,labels,tests)
np.savez(path+'/train'+target,datas=np.array(datas),labels=np.array(labels))
np.savez(path+'/test'+target,tests=np.array(tests))

scaler=MinMaxScaler()
datas=scaler.fit_transform(datas)
tests=scaler.transform(tests)
scaler2=StandardScaler()
datas=scaler2.fit_transform(datas)
tests=scaler2.transform(tests)
np.savez(path+'/train'+target2,datas=np.array(datas),labels=np.array(labels))
np.savez(path+'/test'+target2,tests=np.array(tests))

datas2=part(datas)
tests2=part(tests)
np.savez(path+'/train'+target4,datas=np.array(datas2),labels=np.array(labels))
np.savez(path+'/test'+target4,tests=np.array(tests2))
d=[]
d.extend(datas)
d.extend(tests)
pca=PCA(n_components=numpca)
d2=pca.fit_transform(d)
datas2=d2[:-7]
tests2=d2[-7:]
print len(datas),len(tests)
print len(datas2),len(tests2)
np.savez(path+'/train'+target3,datas=np.array(datas2),labels=np.array(labels))
np.savez(path+'/test'+target3,tests=np.array(tests2))

if path=='./data_task1':
    pca=PCA(n_components=numpca2)
    d2=pca.fit_transform(d)
    datas2=d2[:-7]
    tests2=d2[-7:]
    np.savez(path+'/train'+target3+str(numpca2),datas=np.array(datas2),labels=np.array(labels))
    np.savez(path+'/test'+target3+str(numpca2),tests=np.array(tests2))

    pca=PCA(n_components=numpca3)
    d2=pca.fit_transform(d)
    datas2=d2[:-7]
    tests2=d2[-7:]
    np.savez(path+'/train'+target3+str(numpca3),datas=np.array(datas2),labels=np.array(labels))
    np.savez(path+'/test'+target3+str(numpca3),tests=np.array(tests2))

    pca=PCA(n_components=numpca4)
    d2=pca.fit_transform(d)
    datas2=d2[:-7]
    tests2=d2[-7:]
    np.savez(path+'/train'+target3+str(numpca4),datas=np.array(datas2),labels=np.array(labels))
    np.savez(path+'/test'+target3+str(numpca4),tests=np.array(tests2))
